{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入库和数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sys\n",
    "import seaborn as sns\n",
    "from time import sleep\n",
    "import matplotlib.pyplot as plt\n",
    "on_train=pd.read_csv(r'D:\\Data\\TCForNewComer\\ccf_online_stage1_train\\ccf_online_stage1_train.csv')\n",
    "off_train=pd.read_csv(r'D:\\Data\\TCForNewComer\\ccf_offline_stage1_train\\ccf_offline_stage1_train.csv')\n",
    "# oftid_train=pd.read_csv(r'D:\\Data\\TCForNewComer\\off_train_sameUser_id_test.csv')\n",
    "test=pd.read_csv(r'D:\\Data\\TCForNewComer\\ccf_offline_stage1_test_revised.csv')\n",
    "samplt=pd.read_csv(r'D:\\Data\\TCForNewComer\\sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "off_train=off_train.sort_values(by=['User_id'])\n",
    "off_train.index=np.arange(0,len(off_train),1)\n",
    "off_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test['Date_received']=test['Date_received'].astype(np.int64)\n",
    "test['Date_received']=test['Date_received'].apply(lambda x: str(x))\n",
    "test.index=np.arange(0,len(test),1)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "on_train=on_train.sort_values(by=['User_id'])\n",
    "on_train.index=np.arange(0,len(on_train),1)\n",
    "on_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "col='Merchant_id'\n",
    "a=set(on_train[col].values)\n",
    "b=set(test[col].values)\n",
    "c=set(oftid_train[col].values)\n",
    "print(len(b))\n",
    "print(len(a))\n",
    "print(len(c))\n",
    "\n",
    "print(len(a&b))\n",
    "print(len(b&c))\n",
    "print(len(c-b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 增加一列之前没用优惠券是否购买过相同商品\n",
    "- 可能是日常用品，或者零食\n",
    "- 可能是耐用品，然后买了之后就不会再买"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oftid_train=off_train.copy()\n",
    "indexs=oftid_train[(oftid_train['Coupon_id']=='null')&(oftid_train['Date']!='null')].index\n",
    "odinary=oftid_train.iloc[indexs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oftid_train.drop(indexs,axis=0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oftid_train['ord_buy']=0#增加一列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "odinary.index=np.arange(0,len(odinary),1)\n",
    "oftid_train.index=np.arange(0,len(oftid_train),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#计算同一件商品半年内普通购买次数\n",
    "from ipykernel import kernelapp as app\n",
    "i=0\n",
    "j=10\n",
    "length=len(oftid_train)\n",
    "for index in oftid_train.index:\n",
    "    rate=int(i*100/length)#看进度\n",
    "    if (rate in list(np.arange(10.,120.,10.)))&(rate==j):\n",
    "        print('.'*int(rate/10),rate,'%')\n",
    "        j=j+10\n",
    "\n",
    "    uid=oftid_train.iloc[index]['User_id']\n",
    "    mid=oftid_train.iloc[index]['Merchant_id']\n",
    "    if len(odinary[odinary['User_id']==uid].index)!=0:\n",
    "        for ind in odinary[odinary['User_id']==uid].index:\n",
    "            if odinary.iloc[ind]['Merchant_id']==mid:\n",
    "                oftid_train.loc[index,'ord_buy']+=1\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oftid_train.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\oftid_addordinarybuy_train.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['ord_buy']=0#增加一列\n",
    "test.index=np.arange(0,len(test),1)\n",
    "\n",
    "#计算同一件商品半年内普通购买次数\n",
    "i=0\n",
    "j=10\n",
    "length=len(test)\n",
    "for index in test.index:\n",
    "    rate=int(i*100/length)#看进度\n",
    "    if (rate in list(np.arange(10.,120.,10.)))&(rate==j):\n",
    "        print('.'*int(rate/10),rate,'%')\n",
    "        j=j+10\n",
    "\n",
    "    uid=test.iloc[index]['User_id']\n",
    "    mid=test.iloc[index]['Merchant_id']\n",
    "    if len(odinary[odinary['User_id']==uid].index)!=0:\n",
    "        for ind in odinary[odinary['User_id']==uid].index:\n",
    "            if odinary.iloc[ind]['Merchant_id']==mid:\n",
    "                test.loc[index,'ord_buy']+=1\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\test_addordinarybuy.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 线下领取优惠券张数和不同优惠券的张数和使用数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "or_train=pd.read_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\oftid_addordinarybuy_train.csv')\n",
    "or_test=pd.read_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\test_addordinarybuy.csv')\n",
    "off_train=pd.read_csv(r'D:\\Data\\TCForNewComer\\ccf_offline_stage1_train\\ccf_offline_stage1_train.csv')\n",
    "or_train=or_train.sort_values(by=['User_id'])\n",
    "off_train.index=np.arange(0,len(off_train),1)\n",
    "or_train.index=np.arange(0,len(or_train),1)\n",
    "or_test.index=np.arange(0,len(or_test),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Discount_rate(x):#转化函数\n",
    "    if x.startswith('0'):\n",
    "        return float(x)\n",
    "    else:\n",
    "        return int(x.split(':')[0])-int(x.split(':')[1])\n",
    "    #将满减转化为小数\n",
    "or_train['Discount_rate']=or_train['Discount_rate'].apply(Discount_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ". 10 %-.. 20 %-... 30 %-.... 40 %-..... 50 %-...... 60 %-....... 70 %-........ 80 %-......... 90 %-"
     ]
    }
   ],
   "source": [
    "or_train['Coupon_num']=0                   #领取张数\n",
    "or_train['Coupon_nouse_num']=0             #领取没有使用张数\n",
    "or_train['Coupon_use_num']=0               #使用张数\n",
    "or_train['Coupon_use_prob']=0              #使用概率\n",
    "i=0\n",
    "j=10\n",
    "length=len(set(or_train['User_id'].values))\n",
    "for id in set(or_train['User_id'].values):\n",
    "    rate=int(i*100/length)#看进度\n",
    "    if (rate in list(np.arange(10.,120.,10.)))&(rate==j):\n",
    "        print('.'*int(rate/10),rate,'%',end='-')\n",
    "        j=j+10\n",
    "    \n",
    "    mid_Ser=or_train[or_train['User_id']==id]['Date']\n",
    "    lengths=len(mid_Ser[mid_Ser=='null'])\n",
    "    or_train.loc[mid_Ser.index,'Coupon_num']=len(mid_Ser)\n",
    "    or_train.loc[mid_Ser.index,'Coupon_nouse_num']=lengths\n",
    "    or_train.loc[mid_Ser.index,'Coupon_use_num']=len(mid_Ser)-lengths\n",
    "    or_train.loc[mid_Ser.index,'Coupon_use_prob']=float(len(mid_Ser)-lengths)/len(mid_Ser)\n",
    "    i+=1\n",
    "or_train.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\or_train.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ". 10 %-.. 20 %-... 30 %-.... 40 %-..... 50 %-...... 60 %-....... 70 %-........ 80 %-......... 90 %-"
     ]
    }
   ],
   "source": [
    "or_test['Coupon_num']=0                   #领取张数\n",
    "or_test['Coupon_nouse_num']=0             #领取没有使用张数\n",
    "or_test['Coupon_use_num']=0               #使用张数\n",
    "or_test['Coupon_use_prob']=0              #使用概率\n",
    "i=0\n",
    "j=10\n",
    "length=len(set(or_test['User_id'].values))\n",
    "for id in set(or_test['User_id'].values):\n",
    "    rate=int(i*100/length)#看进度\n",
    "    if (rate in list(np.arange(10.,120.,10.)))&(rate==j):\n",
    "        print('.'*int(rate/10),rate,'%',end='-')\n",
    "        j=j+10\n",
    "    \n",
    "    mid_Ser=or_train[or_train['User_id']==id]['Date']\n",
    "    if len(mid_Ser)!=0:\n",
    "        lengths=len(mid_Ser[mid_Ser=='null'])\n",
    "        test_Ser=or_test[or_test['User_id']==id]\n",
    "        or_test.loc[test_Ser.index,'Coupon_num']=len(mid_Ser)\n",
    "        or_test.loc[test_Ser.index,'Coupon_nouse_num']=lengths\n",
    "        or_test.loc[test_Ser.index,'Coupon_use_num']=len(mid_Ser)-lengths\n",
    "        or_test.loc[test_Ser.index,'Coupon_use_prob']=float(len(mid_Ser)-lengths)/len(mid_Ser)\n",
    "    i+=1\n",
    "or_test.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\or_test.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 使用优惠券面额大于100的次数和小于100的次数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "oc_train=pd.read_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\train.csv')\n",
    "oc_test=pd.read_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\test.csv')\n",
    "oc_train=oc_train.sort_values(by=['User_id'])\n",
    "oc_train.index=np.arange(0,len(oc_train),1)\n",
    "oc_test.index=np.arange(0,len(oc_test),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ". 10 %-.. 20 %-... 30 %-.... 40 %-..... 50 %-...... 60 %-....... 70 %-........ 80 %-......... 90 %-"
     ]
    }
   ],
   "source": [
    "oc_train['Coupon_greaterthan100']=0                   #使用面额大于100的张数\n",
    "oc_train['Coupon_lessthan100']=0                      #使用面额小于100的张数\n",
    "oc_train['Coupon_lep100']=0                          #使用面额大于100的张数占使用张数的比重\n",
    "oc_train['Coupon_grp100']=0                          #使用面额小于100的张数占使用张数的比重\n",
    "oc_train['Avg_saler_Coupon']=0                       #核销每个商家多少张优惠券\n",
    "oc_train['Avg_distance']=0                          #用户核销优惠券的平均距离\n",
    "oc_train['shop_count']=0                            #核销商家数量\n",
    "oc_train['shop_pro']=0                              #商家优惠券占总的比例\n",
    "oc_train['Coupon_diffcount']=0                      #不同优惠券的数量\n",
    "oc_train['Coupon_diffpro']=0                        #不同优惠券的占比\n",
    "oc_train['same_Coupon_count']=0                      #相同优惠券领取的次数\n",
    "i=0\n",
    "j=10\n",
    "length=len(set(oc_train[oc_train['Date']==1]['User_id'].values))\n",
    "for id in set(oc_train[oc_train['Date']==1]['User_id'].values):\n",
    "    rate=int(i*100/length)#看进度\n",
    "    if (rate in list(np.arange(10.,120.,10.)))&(rate==j):\n",
    "        print('.'*int(rate/10),rate,'%',end='-')\n",
    "        j=j+10\n",
    "    \n",
    "    discount_rate=oc_train[(oc_train['User_id']==id)&(oc_train['Date']==1)]['Discount_rate']\n",
    "    indexs=oc_train[oc_train['User_id']==id].index\n",
    "    gr100=len(discount_rate[discount_rate>100])\n",
    "    le100=len(discount_rate[discount_rate<100])\n",
    "    use_count=oc_train.loc[discount_rate.index,'Coupon_use_num'].max()\n",
    "    oc_train.loc[indexs,'Coupon_greaterthan100']=gr100\n",
    "    oc_train.loc[indexs,'Coupon_lessthan100']=le100\n",
    "    oc_train.loc[indexs,'Coupon_grp100']=gr100/use_count\n",
    "    oc_train.loc[indexs,'Coupon_lep100']=le100/use_count\n",
    "    \n",
    "    customer=oc_train[(oc_train['User_id']==id)&(oc_train['Date']==1)]\n",
    "    oc_train.loc[indexs,'shop_count']=len(set(customer['Distance'].values))\n",
    "    oc_train.loc[indexs,'Coupon_diffcount']=len(set(customer['Discount_rate'].values))\n",
    "    for dis in set(customer['Distance'].values):                     #不同的商家\n",
    "        cus=customer[customer['Distance']==dis]\n",
    "        indexss=oc_train[(oc_train['User_id']==id)&(oc_train['Distance']==dis)].index\n",
    "        oc_train.loc[indexss,'Avg_saler_Coupon']=len(cus)\n",
    "        oc_train.loc[indexss,'shop_pro']=len(cus)/len(customer)\n",
    "        for rat in set(customer['Discount_rate'].values):             #不同优惠额度\n",
    "            Rate=customer[customer['Discount_rate']==rat]\n",
    "            indexsss=oc_train[(oc_train['User_id']==id)&(oc_train['Distance']==dis)&(oc_train['Discount_rate']==rat)].index\n",
    "            oc_train.loc[indexsss,'Coupon_diffpro']=float(len(Rate))/len(cus)\n",
    "    \n",
    "    for coupid in set(oc_train[oc_train['User_id']==id]['Coupon_id'].values):\n",
    "        indexx=oc_train[(oc_train['User_id']==id)&(oc_train['Coupon_id']==coupid)].index\n",
    "        oc_train.loc[indexx,'same_Coupon_count']=len(indexx)\n",
    "        \n",
    "    oc_train.loc[indexs,'Avg_distance']=customer['Distance'].mean()\n",
    "    i+=1\n",
    "oc_train.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\oc_train.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ". 10 %-.. 20 %-... 30 %-.... 40 %-..... 50 %-...... 60 %-....... 70 %-........ 80 %-......... 90 %-"
     ]
    }
   ],
   "source": [
    "oc_test['Coupon_greaterthan100']=0                   #使用面额大于100的张数\n",
    "oc_test['Coupon_lessthan100']=0                      #使用面额小于100的张数\n",
    "oc_test['Coupon_lep100']=0                          #使用面额大于100的张数占使用张数的比重\n",
    "oc_test['Coupon_grp100']=0                          #使用面额小于100的张数占使用张数的比重\n",
    "oc_test['Avg_saler_Coupon']=0                       #核销每个商家多少张优惠券\n",
    "oc_test['Avg_distance']=0                          #用户核销优惠券的平均距离\n",
    "oc_test['shop_count']=0                            #核销商家数量\n",
    "oc_test['shop_pro']=0                              #商家优惠券占总的比例\n",
    "oc_test['Coupon_diffcount']=0                      #不同优惠券的数量\n",
    "oc_test['Coupon_diffpro']=0                        #不同优惠券的占比\n",
    "oc_test['same_Coupon_count']=0                      #相同优惠券领取的次数\n",
    "i=0\n",
    "j=10\n",
    "length=len(set(oc_test['User_id'].values))\n",
    "for id in set(oc_test['User_id'].values):\n",
    "    rate=int(i*100/length)#看进度\n",
    "    if (rate in list(np.arange(10.,120.,10.)))&(rate==j):\n",
    "        print('.'*int(rate/10),rate,'%',end='-')\n",
    "        j=j+10\n",
    "    \n",
    "    discount_rate=oc_train[(oc_train['User_id']==id)&(oc_train['Date']==1)]['Discount_rate']\n",
    "    indexs=oc_test[oc_test['User_id']==id].index\n",
    "    gr100=len(discount_rate[discount_rate>100])\n",
    "    le100=len(discount_rate[discount_rate<100])\n",
    "    use_count=oc_train.loc[discount_rate.index,'Coupon_use_num'].max()\n",
    "    oc_test.loc[indexs,'Coupon_greaterthan100']=gr100\n",
    "    oc_test.loc[indexs,'Coupon_lessthan100']=le100\n",
    "    if len(discount_rate)!=0:\n",
    "        oc_test.loc[indexs,'Coupon_grp100']=gr100/use_count\n",
    "        oc_test.loc[indexs,'Coupon_lep100']=le100/use_count\n",
    "    \n",
    "    customer=oc_train[(oc_train['User_id']==id)&(oc_train['Date']==1)]\n",
    "    oc_test.loc[indexs,'shop_count']=len(set(customer['Distance'].values))\n",
    "    oc_test.loc[indexs,'Coupon_diffcount']=len(set(customer['Discount_rate'].values))\n",
    "    for dis in set(oc_test[(oc_test['User_id']==id)]['Distance'].values):                     #不同的商家\n",
    "        cus=customer[customer['Distance']==dis]\n",
    "        indexss=oc_test[(oc_test['User_id']==id)&(oc_test['Distance']==dis)].index\n",
    "        oc_test.loc[indexss,'Avg_saler_Coupon']=len(cus)\n",
    "        if len(customer)!=0:\n",
    "            oc_test.loc[indexss,'shop_pro']=float(len(cus))/len(customer)\n",
    "        \n",
    "        for rat in set(oc_test[(oc_test['User_id']==id)]['Discount_rate'].values):             #不同优惠额度\n",
    "            Rate=customer[customer['Discount_rate']==rat]\n",
    "            indexsss=oc_test[(oc_test['User_id']==id)&(oc_test['Distance']==dis)&(oc_test['Discount_rate']==rat)].index\n",
    "            if len(cus)!=0:\n",
    "                oc_test.loc[indexsss,'Coupon_diffpro']=float(len(Rate))/len(cus)\n",
    "    \n",
    "    for coupid in set(oc_test[oc_test['User_id']==id]['Coupon_id'].values):\n",
    "        indexx=oc_test[(oc_test['User_id']==id)&(oc_test['Coupon_id']==coupid)].index\n",
    "        oc_test.loc[indexx,'same_Coupon_count']=len(indexx)\n",
    "    if len(customer['Distance'])!=0:\n",
    "        oc_test.loc[indexs,'Avg_distance']=customer['Distance'].mean()\n",
    "    i+=1\n",
    "oc_test.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\oc_test.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "##  商家提供的优惠券种类和张数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 算有几家店铺，店铺中的商品种类，店铺的优惠券种类和张数，顾客经常使用优惠券的商店，对店铺进行编码，优惠券领取的张数，优惠券使用的张数和占比，发放优惠券的平均面额"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "os_train=pd.read_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\oc_train.csv')\n",
    "os_test=pd.read_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\oc_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shopn(id):                        #计算店铺数量\n",
    "    for mid in  id:\n",
    "        indexs=[]\n",
    "        indexs.extend(list(os_train[os_train['Merchant_id']==mid].index))\n",
    "        avg=os_train[os_train['Merchant_id']==mid]['Discount_rate'].mean()\n",
    "\n",
    "        for uid in set(os_train[os_train['Merchant_id']==mid]['User_id'].values):\n",
    "            dis=os_train[(os_train['Merchant_id']==mid)&(os_train['User_id']==uid)]['Distance'].max()\n",
    "            merchantid=set()\n",
    "            for ind in os_train[(os_train['User_id']==uid)&(os_train['Distance']==int(dis))].index:\n",
    "                if (abs(os_train.loc[ind,'Discount_rate']-avg)<100):\n",
    "                    indexs.extend([ind])\n",
    "                    if ind not in indexs:\n",
    "                        merchantid.add(os_train.loc[ind,'Merchant_id'])\n",
    "            if len(merchantid)!=0:\n",
    "                indexs.extend(shopn(merchantid)) \n",
    "    return indexs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 增加用券在线购买和普通购买、领取优惠券、点击的列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test=pd.read_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\test_addordinarybuy.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oftid_train['online_buy']=0#增加一列\n",
    "oftid_train['online_col']=0\n",
    "oftid_train['online_click']=0\n",
    "test['online_buy']=0#增加一列\n",
    "test['online_col']=0\n",
    "test['online_click']=0\n",
    "# odinary.index=np.arange(0,len(odinary),1)\n",
    "# ontid_train.index=np.arange(0,len(ontid_train),1)\n",
    "oftid_train.index=np.arange(0,len(oftid_train),1)\n",
    "test.index=np.arange(0,len(test),1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "i=0\n",
    "j=10\n",
    "length=len(oftid_train)\n",
    "for index in oftid_train.index:\n",
    "    rate=int(i*100/length)#看进度\n",
    "    if (rate in list(np.arange(10.,120.,10.)))&(rate==j):\n",
    "        print('.'*int(rate/10),rate,'%')\n",
    "        j=j+10\n",
    "\n",
    "    uid=oftid_train.iloc[index]['User_id']\n",
    "    if len(on_train[on_train['User_id']==uid].index)!=0:\n",
    "        for ind in on_train[on_train['User_id']==uid].index:\n",
    "            if on_train.iloc[ind]['Action']==1:\n",
    "                oftid_train.loc[index,'online_buy']+=1\n",
    "            elif on_train.iloc[ind]['Action']==2:\n",
    "                oftid_train.loc[index,'online_col']+=1\n",
    "            else:\n",
    "                oftid_train.loc[index,'online_click']+=1\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oftid_train.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\ofon_coupod_buy_train.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "i=0\n",
    "j=10\n",
    "length=len(test)\n",
    "for index in test.index:\n",
    "    rate=int(i*100/length)#看进度\n",
    "    if (rate in list(np.arange(10.,120.,10.)))&(rate==j):\n",
    "        print('.'*int(rate/10),rate,'%')\n",
    "        j=j+10\n",
    "\n",
    "    uid=test.iloc[index]['User_id']\n",
    "    if len(on_train[on_train['User_id']==uid].index)!=0:\n",
    "        for ind in on_train[on_train['User_id']==uid].index:\n",
    "            if on_train.iloc[ind]['Action']==1:\n",
    "                test.loc[index,'online_buy']+=1\n",
    "            elif on_train.iloc[ind]['Action']==2:\n",
    "                test.loc[index,'online_col']+=1\n",
    "            else:\n",
    "                test.loc[index,'online_click']+=1\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\test_ofon_coupod_buy.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "off_train=oftid_train.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导入增加几列以后的数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sys\n",
    "import seaborn as sns\n",
    "from time import sleep\n",
    "import matplotlib.pyplot as plt\n",
    "off_train=pd.read_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\or_train.csv')\n",
    "test=pd.read_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\or_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Merchant_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Discount_rate</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>Date</th>\n",
       "      <th>ord_buy</th>\n",
       "      <th>Coupon_num</th>\n",
       "      <th>Coupon_nouse_num</th>\n",
       "      <th>Coupon_use_num</th>\n",
       "      <th>Coupon_use_prob</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>1433</td>\n",
       "      <td>8735</td>\n",
       "      <td>25.00</td>\n",
       "      <td>10</td>\n",
       "      <td>20160214</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>1469</td>\n",
       "      <td>2902</td>\n",
       "      <td>0.95</td>\n",
       "      <td>10</td>\n",
       "      <td>20160607</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35</td>\n",
       "      <td>3381</td>\n",
       "      <td>1807</td>\n",
       "      <td>270.00</td>\n",
       "      <td>0</td>\n",
       "      <td>20160130</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35</td>\n",
       "      <td>3381</td>\n",
       "      <td>9776</td>\n",
       "      <td>5.00</td>\n",
       "      <td>0</td>\n",
       "      <td>20160129</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35</td>\n",
       "      <td>3381</td>\n",
       "      <td>11951</td>\n",
       "      <td>180.00</td>\n",
       "      <td>0</td>\n",
       "      <td>20160129</td>\n",
       "      <td>null</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_id  Merchant_id  Coupon_id  Discount_rate Distance Date_received  \\\n",
       "0        4         1433       8735          25.00       10      20160214   \n",
       "1        4         1469       2902           0.95       10      20160607   \n",
       "2       35         3381       1807         270.00        0      20160130   \n",
       "3       35         3381       9776           5.00        0      20160129   \n",
       "4       35         3381      11951         180.00        0      20160129   \n",
       "\n",
       "   Date  ord_buy  Coupon_num  Coupon_nouse_num  Coupon_use_num  \\\n",
       "0  null        0           2                 2               0   \n",
       "1  null        0           2                 2               0   \n",
       "2  null        0           4                 4               0   \n",
       "3  null        0           4                 4               0   \n",
       "4  null        0           4                 4               0   \n",
       "\n",
       "   Coupon_use_prob  \n",
       "0              0.0  \n",
       "1              0.0  \n",
       "2              0.0  \n",
       "3              0.0  \n",
       "4              0.0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "off_train=off_train.sort_values(by=['User_id'])\n",
    "off_train['Date_received']=off_train['Date_received'].apply(lambda x: str(x))\n",
    "off_train.index=np.arange(0,len(off_train),1)\n",
    "off_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_id</th>\n",
       "      <th>Merchant_id</th>\n",
       "      <th>Coupon_id</th>\n",
       "      <th>Discount_rate</th>\n",
       "      <th>Distance</th>\n",
       "      <th>Date_received</th>\n",
       "      <th>ord_buy</th>\n",
       "      <th>Coupon_num</th>\n",
       "      <th>Coupon_nouse_num</th>\n",
       "      <th>Coupon_use_num</th>\n",
       "      <th>Coupon_use_prob</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4129537</td>\n",
       "      <td>450</td>\n",
       "      <td>9983</td>\n",
       "      <td>25.0</td>\n",
       "      <td>1</td>\n",
       "      <td>20160712</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6949378</td>\n",
       "      <td>1300</td>\n",
       "      <td>3429</td>\n",
       "      <td>25.0</td>\n",
       "      <td>null</td>\n",
       "      <td>20160706</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2166529</td>\n",
       "      <td>7113</td>\n",
       "      <td>6928</td>\n",
       "      <td>180.0</td>\n",
       "      <td>5</td>\n",
       "      <td>20160727</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2166529</td>\n",
       "      <td>7113</td>\n",
       "      <td>1808</td>\n",
       "      <td>90.0</td>\n",
       "      <td>5</td>\n",
       "      <td>20160727</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6172162</td>\n",
       "      <td>7605</td>\n",
       "      <td>6500</td>\n",
       "      <td>29.0</td>\n",
       "      <td>2</td>\n",
       "      <td>20160708</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_id  Merchant_id  Coupon_id  Discount_rate Distance Date_received  \\\n",
       "0  4129537          450       9983           25.0        1      20160712   \n",
       "1  6949378         1300       3429           25.0     null      20160706   \n",
       "2  2166529         7113       6928          180.0        5      20160727   \n",
       "3  2166529         7113       1808           90.0        5      20160727   \n",
       "4  6172162         7605       6500           29.0        2      20160708   \n",
       "\n",
       "   ord_buy  Coupon_num  Coupon_nouse_num  Coupon_use_num  Coupon_use_prob  \n",
       "0        0           2                 2               0              0.0  \n",
       "1        0           1                 0               1              1.0  \n",
       "2        0           1                 1               0              0.0  \n",
       "3        0           1                 1               0              0.0  \n",
       "4        0           1                 1               0              0.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def Discount_rate(x):#转化函数\n",
    "    if x.startswith('0'):\n",
    "        return float(x)\n",
    "    else:\n",
    "        return int(x.split(':')[0])-int(x.split(':')[1])\n",
    "#将满减转化为小数\n",
    "test['Discount_rate']=test['Discount_rate'].apply(Discount_rate)\n",
    "test['Date_received']=test['Date_received'].astype(np.int64)\n",
    "test['Date_received']=test['Date_received'].apply(lambda x: str(x))\n",
    "test.index=np.arange(0,len(test),1)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 选取间隔日期大于15天，标记为负样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "p_train=off_train[off_train['Date']!='null'].copy()#正样本\n",
    "n_train=off_train[off_train['Date']=='null'].copy()#负样本\n",
    "p_train.index=np.arange(0,len(p_train),1)\n",
    "n_train.index=np.arange(0,len(n_train),1)\n",
    "#转换为时间格式\n",
    "p_train['Date_received_copy']=p_train['Date_received'].copy()\n",
    "\n",
    "p_train['Date_received_copy']=pd.to_datetime(p_train['Date_received_copy'])\n",
    "p_train['Date']=pd.to_datetime(p_train['Date'])\n",
    "# n_train['Date_received']=pd.to_datetime(n_train['Date_received'])\n",
    "# test['Date_received']=pd.to_datetime(test['Date_received'])\n",
    "\n",
    "p_train['date']=(p_train['Date']-p_train['Date_received_copy']).astype('timedelta64[D]')\n",
    "p_train.loc[p_train[p_train['date']<=15].index,'date']=1\n",
    "p_train.loc[p_train[p_train['date']>15].index,'date']=0\n",
    "p_train['Date']=p_train['date'].copy()\n",
    "p_train.drop(['date','Date_received_copy'],axis=1,inplace=True)\n",
    "p_train['Date']=p_train['Date'].astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#转换为星期几x.weekday()+1\n",
    "test['Date_received']=(test['Date_received'].apply(lambda x: int(x[5:8]))).copy()\n",
    "#正样本为1，负样本为0\n",
    "n_train['Date']=0\n",
    "train=pd.concat([p_train,n_train],axis=0)\n",
    "train['Date_received']=(train['Date_received'].apply(lambda x: int(x[5:8]))).copy()\n",
    "train['Date_received']=train['Date_received'].astype(np.int64)\n",
    "train=train.sort_values(by=['User_id'])\n",
    "train.index=np.arange(0,len(train),1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 转化满减为小数折扣"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Discount_rate(x):#转化函数\n",
    "    if x.startswith('0'):\n",
    "        return float(x)\n",
    "    else:\n",
    "        return int(x.split(':')[0])-int(x.split(':')[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train=train.replace('null',np.nan)\n",
    "test=test.replace('null',np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#将满减转化为小数\n",
    "train['Discount_rate']=train['Discount_rate'].apply(Discount_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train=train.fillna(method='ffill')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['Distance']=train['Distance'].astype(np.int64)\n",
    "train['Coupon_id']=train['Coupon_id'].astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1053282 entries, 0 to 1053281\n",
      "Data columns (total 12 columns):\n",
      "User_id             1053282 non-null int64\n",
      "Merchant_id         1053282 non-null int64\n",
      "Coupon_id           1053282 non-null int64\n",
      "Discount_rate       1053282 non-null float64\n",
      "Distance            1053282 non-null int64\n",
      "Date_received       1053282 non-null int64\n",
      "Date                1053282 non-null int64\n",
      "ord_buy             1053282 non-null int64\n",
      "Coupon_num          1053282 non-null int64\n",
      "Coupon_nouse_num    1053282 non-null int64\n",
      "Coupon_use_num      1053282 non-null int64\n",
      "Coupon_use_prob     1053282 non-null float64\n",
      "dtypes: float64(2), int64(10)\n",
      "memory usage: 104.5 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 不同类型数据的影响"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['Merchant_id']=train['Merchant_id'].astype(np.object)\n",
    "train['User_id']=train['User_id'].astype(np.object)\n",
    "train['Date_received']=train['Date_received'].astype('category')\n",
    "test['Merchant_id']=test['Merchant_id'].astype(np.object)\n",
    "test['Coupon_id']=test['Coupon_id'].astype(np.object)\n",
    "test['Distance']=test['Distance'].astype(np.object)\n",
    "test['Date_received']=test['Date_received'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['User_id']=test['User_id'].astype(np.object)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 填充测试集的空缺值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test=test.fillna(method='ffill')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['Discount_rate']=test['Discount_rate'].apply(Discount_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['Merchant_id']=test['Merchant_id'].astype(np.int64)\n",
    "test['Coupon_id']=test['Coupon_id'].astype(np.int64)\n",
    "test['Distance']=test['Distance'].astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 113640 entries, 0 to 113639\n",
      "Data columns (total 11 columns):\n",
      "User_id             113640 non-null int64\n",
      "Merchant_id         113640 non-null int64\n",
      "Coupon_id           113640 non-null int64\n",
      "Discount_rate       113640 non-null float64\n",
      "Distance            113640 non-null int64\n",
      "Date_received       113640 non-null int64\n",
      "ord_buy             113640 non-null int64\n",
      "Coupon_num          113640 non-null int64\n",
      "Coupon_nouse_num    113640 non-null int64\n",
      "Coupon_use_num      113640 non-null int64\n",
      "Coupon_use_prob     113640 non-null float64\n",
      "dtypes: float64(2), int64(9)\n",
      "memory usage: 10.4 MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\train.csv',index=False)\n",
    "test.to_csv(r'D:\\Data\\TCForNewComer\\deal\\ALLuserid\\test.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#行最大最小归一化\n",
    "XX=train.copy()\n",
    "for col in ['Merchant_id','Coupon_id']:\n",
    "    max=XX[col].max()\n",
    "    min=XX[col].min()\n",
    "    XX[col]=XX[col].apply(lambda x: ((x-min)/(max-min)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "XX.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cor=XX.corr()\n",
    "cor[cor<0.01]=0\n",
    "cor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 距离统计分布"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import seaborn as sns \n",
    "%matplotlib inline\n",
    "\n",
    "sns.countplot(x='Distance',hue='Date',data=train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "sns.countplot(x='Date_received',hue='Date',data=train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import seaborn as sns \n",
    "%matplotlib inline\n",
    "plt.figure(figsize=(10,6))\n",
    "sns.countplot(x='Discount_rate',hue='Date',data=train)\n",
    "plt.xticks(rotation=90)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=set(train['Coupon_id'].values)\n",
    "c=set(test['Coupon_id'].values)\n",
    "len(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corr=train.corr()\n",
    "corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "combine=[train,test]\n",
    "for data in combine:\n",
    "    data.drop(['Date_received'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a=set(train['Merchant_id'].values)\n",
    "c=set(test['Merchant_id'].values)\n",
    "len(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(a&c)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "K=3\n",
    "from sklearn.metrics import roc_auc_score,auc\n",
    "from time import sleep\n",
    "import winsound\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "# kf = KFold(n_splits = K,random_state = 1,shuffle = True)\n",
    "kf=StratifiedKFold(n_splits = K,random_state = 90,shuffle = True)\n",
    "X=train.drop(['Date'],axis=1)\n",
    "y=train['Date']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X=X.drop(['online_click'],axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 归一化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "x_predict=test\n",
    "# 1. min max scaler\n",
    "min_max_scaler = MinMaxScaler()\n",
    "x_predict_min_max_scaled = min_max_scaler.fit_transform(x_predict)\n",
    "\n",
    "# 2. standard scaler\n",
    "std_scaler = StandardScaler()\n",
    "x_predict_std_scaled = std_scaler.fit_transform(x_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 1. min max scaler\n",
    "min_max_scaler = MinMaxScaler()\n",
    "# x_train_min_max_scaled = min_max_scaler.fit_transform(x_train)\n",
    "# x_test_min_max_scaled = min_max_scaler.transform(x_test)\n",
    "X_test_min_max_scaled=min_max_scaler.fit_transform(X)#整个数据集\n",
    "# 2. standard scaler\n",
    "std_scaler = StandardScaler()\n",
    "# x_train_std_scaled = std_scaler.fit_transform(x_train)\n",
    "# x_test_std_scaled = std_scaler.transform(x_test)\n",
    "X_test_std_scaled=std_scaler.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "#全部  0.5552 learning_rate=0.09\n",
    "model=xgb.XGBClassifier(n_estimators=200,\n",
    "                        max_depth=2,#6\n",
    "                        objective=\"binary:logistic\",\n",
    "                        learning_rate=0.09, \n",
    "                        subsample=.8,\n",
    "                        min_child_weight=6,\n",
    "                        colsample_bytree=.4,\n",
    "                        scale_pos_weight=1.6,\n",
    "                        gamma=9,\n",
    "                        seed=100,\n",
    "                        reg_alpha=8,\n",
    "                        reg_lambda=1.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Avg_auc(pred):\n",
    "    aucc=0\n",
    "    i=0\n",
    "    df=train.copy()\n",
    "    df['y_valid_pred']=pred\n",
    "    for j in list(set(df['Coupon_id'].values)):\n",
    "        df_1=df[df['Coupon_id']==j]\n",
    "        if len(np.unique(df_1['Date']))==1:\n",
    "            continue\n",
    "        aucc=aucc+roc_auc_score(df_1['Date'],df_1['y_valid_pred'])\n",
    "        i+=1\n",
    "    aucc=aucc/i\n",
    "    df.drop(['y_valid_pred'],axis=1,inplace=True)\n",
    "    return aucc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "model = GaussianNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "model=KNeighborsClassifier(n_neighbors=5)\n",
    "model.fit(X,y)\n",
    "pred=model.predict_proba(X)[:,1]\n",
    "print('Avg_auc for full training set:',Avg_auc(pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier#星期0.626\n",
    "model=AdaBoostClassifier(n_estimators=400,learning_rate=1,random_state=100)\n",
    "model.fit(X,y)\n",
    "pred=model.predict_proba(X)[:,1]\n",
    "pred_test=model.predict_proba(test)[:,1]\n",
    "print('Avg_auc for full training set:',Avg_auc(pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y_test_pred=0\n",
    "y_valid_pred=0*y\n",
    "fit_model=[]\n",
    "# XMIN=pd.DataFrame(X_test_std_scaled,columns=['Merchant_id', 'Coupon_id', 'Discount_rate', 'Distance', 'Date_received'])\n",
    "for i,(train_index,test_index) in enumerate(kf.split(X,y)):\n",
    "    y_train,y_valid=y.iloc[train_index],y.iloc[test_index]\n",
    "    x_train,x_valid=X.iloc[train_index],X.iloc[test_index]\n",
    "    print('\\nFlod:',i,end=':')\n",
    "    fitted_model=model.fit(x_train,y_train)\n",
    "    pred= fitted_model.predict_proba(x_valid)[:,1]\n",
    "    y_valid_pred.iloc[test_index]=pred\n",
    "    print(pd.DataFrame(pred).head(1))\n",
    "    \n",
    "    y_test_pred+=fitted_model.predict_proba(test)[:,1]\n",
    "    fit_model.append(fitted_model)\n",
    "    winsound.Beep(600,1000)\n",
    "    sleep(1)\n",
    "y_test_pred/=K\n",
    "print('Avg_auc for full training set:',Avg_auc(y_valid_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tt=pd.read_csv(r'D:\\Data\\TCForNewComer\\ccf_offline_stage1_test_revised.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub=pd.DataFrame()\n",
    "sub['User_id']=tt['User_id']\n",
    "sub['Coupon_id']=tt['Coupon_id']\n",
    "sub['Date_received']=tt['Date_received']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub['Probability']=y_test_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub.to_csv(r'D:\\Data\\TCForNewComer\\ALLuseridS1pred_result.csv',float_format='%.1f',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train['a']=np.random.normal(len(y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print('Avg_auc for full training set:',Avg_auc(y_valid_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score\n",
    "y_true = np.array([0, 1, 0, 1])\n",
    "y_scores = np.array([0.1, 0.4, 0.35, 0.8])\n",
    "y_true.astype(np.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
